{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "from collections import defaultdict\n",
    "import math\n",
    "import operator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "函数说明:创建数据样本\n",
    "Returns:\n",
    "    dataset - 实验样本切分的词条\n",
    "    classVec - 类别标签向量\n",
    "\"\"\"\n",
    "def loadDataSet():\n",
    "    dataset = [ ['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],    # 切分的词条\n",
    "                   ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],\n",
    "                   ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],\n",
    "                   ['stop', 'posting', 'stupid', 'worthless', 'garbage'],\n",
    "                   ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],\n",
    "                   ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid'] ]\n",
    "    classVec = [0, 1, 0, 1, 0, 1]  # 类别标签向量，1代表好，0代表不好\n",
    "    return dataset, classVec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "函数说明：特征选择TF-IDF算法\n",
    "Parameters:\n",
    "     list_words:词列表\n",
    "Returns:\n",
    "     dict_feature_select:特征选择词字典\n",
    "\"\"\"\n",
    "def feature_select(list_words):\n",
    "    #总词频统计\n",
    "    doc_frequency=defaultdict(int)\n",
    "    for word_list in list_words:\n",
    "        for i in word_list:\n",
    "            doc_frequency[i]+=1\n",
    "#     print('doc_frequency：',doc_frequency)\n",
    "    print('doc_frequency的长度：',len(doc_frequency))\n",
    "    #计算每个词的TF值\n",
    "    sum_doc_frequency=0\n",
    "    for i in doc_frequency:\n",
    "        sum_doc_frequency=sum_doc_frequency+doc_frequency[i]\n",
    "    print('sum_doc_frequency',sum_doc_frequency)\n",
    "    word_tf={}  #存储每个词的tf值\n",
    "    for i in doc_frequency:\n",
    "        word_tf[i]=doc_frequency[i]/sum_doc_frequency\n",
    "    #计算每个词的IDF值\n",
    "    doc_num=len(list_words)\n",
    "    word_idf={} #存储每个词的idf值\n",
    "    word_doc=defaultdict(int) #存储包含该词的文档数\n",
    "    for i in doc_frequency:\n",
    "        for j in list_words:\n",
    "            if i in j:\n",
    "                word_doc[i]+=1\n",
    "    for i in doc_frequency:\n",
    "        word_idf[i]=math.log(doc_num/(word_doc[i]+1))\n",
    " \n",
    "    #计算每个词的TF*IDF的值\n",
    "    word_tf_idf={}\n",
    "    for i in doc_frequency:\n",
    "        word_tf_idf[i]=word_tf[i]*word_idf[i]\n",
    "    # 对字典按值由大到小排序\n",
    "    dict_feature_select=sorted(word_tf_idf.items(),key=operator.itemgetter(1),reverse=True)\n",
    "    return dict_feature_select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_values([3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 3, 2, 1, 3, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1])\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for /: 'int' and 'dict_values'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-90-8b1ffee80817>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mdata_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlabel_list\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mloadDataSet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#加载数据\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mfeatures\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeature_select\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata_list\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#所有词的TF-IDF值\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-89-1d7e48c96844>\u001b[0m in \u001b[0;36mfeature_select\u001b[1;34m(list_words)\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc_frequency\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdoc_frequency\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 26\u001b[1;33m         \u001b[0mword_tf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdoc_frequency\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc_frequency\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     27\u001b[0m     \u001b[1;31m#计算每个词的IDF值\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m     \u001b[0mdoc_num\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist_words\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: unsupported operand type(s) for /: 'int' and 'dict_values'"
     ]
    }
   ],
   "source": [
    "data_list,label_list=loadDataSet() #加载数据\n",
    "features=feature_select(data_list) #所有词的TF-IDF值\n",
    "print(features)\n",
    "print(len(features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('to', 0.0322394037469742),\n",
       " ('stop', 0.0322394037469742),\n",
       " ('worthless', 0.0322394037469742),\n",
       " ('my', 0.028288263356383563),\n",
       " ('dog', 0.028288263356383563),\n",
       " ('him', 0.028288263356383563),\n",
       " ('stupid', 0.028288263356383563),\n",
       " ('has', 0.025549122992281622),\n",
       " ('flea', 0.025549122992281622),\n",
       " ('problems', 0.025549122992281622),\n",
       " ('help', 0.025549122992281622),\n",
       " ('please', 0.025549122992281622),\n",
       " ('maybe', 0.025549122992281622),\n",
       " ('not', 0.025549122992281622),\n",
       " ('take', 0.025549122992281622),\n",
       " ('park', 0.025549122992281622),\n",
       " ('dalmation', 0.025549122992281622),\n",
       " ('is', 0.025549122992281622),\n",
       " ('so', 0.025549122992281622),\n",
       " ('cute', 0.025549122992281622),\n",
       " ('I', 0.025549122992281622),\n",
       " ('love', 0.025549122992281622),\n",
       " ('posting', 0.025549122992281622),\n",
       " ('garbage', 0.025549122992281622),\n",
       " ('mr', 0.025549122992281622),\n",
       " ('licks', 0.025549122992281622),\n",
       " ('ate', 0.025549122992281622),\n",
       " ('steak', 0.025549122992281622),\n",
       " ('how', 0.025549122992281622),\n",
       " ('quit', 0.025549122992281622),\n",
       " ('buying', 0.025549122992281622),\n",
       " ('food', 0.025549122992281622)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输出x_train文本向量：\n",
      "[[0.70710678 0.         0.70710678 0.         0.         0.\n",
      "  0.         0.         0.         0.        ]\n",
      " [0.         0.3349067  0.         0.44036207 0.         0.44036207\n",
      "  0.44036207 0.44036207 0.         0.3349067 ]\n",
      " [0.         0.22769009 0.         0.         0.89815533 0.\n",
      "  0.         0.         0.29938511 0.22769009]]\n",
      "输出x_test文本向量：\n",
      "[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    " \n",
    "x_train = ['TF-IDF 主要 思想 是','算法 一个 重要 特点 可以 脱离 语料库 背景',\n",
    "           '如果 一个 网页 被 很多 其他 网页 链接 说明 网页 重要']\n",
    "x_test=['原始 文本 进行 标记','主要 思想']\n",
    " \n",
    "#该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频\n",
    "vectorizer = CountVectorizer(max_features=10)\n",
    "#该类会统计每个词语的tf-idf权值\n",
    "tf_idf_transformer = TfidfTransformer()\n",
    "#将文本转为词频矩阵并计算tf-idf\n",
    "tf_idf = tf_idf_transformer.fit_transform(vectorizer.fit_transform(x_train))\n",
    "#将tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重\n",
    "x_train_weight = tf_idf.toarray()\n",
    " \n",
    "#对测试集进行tf-idf权重计算\n",
    "tf_idf = tf_idf_transformer.transform(vectorizer.transform(x_test))\n",
    "x_test_weight = tf_idf.toarray()  # 测试集TF-IDF权重矩阵\n",
    " \n",
    "print('输出x_train文本向量：')\n",
    "print(x_train_weight)\n",
    "print('输出x_test文本向量：')\n",
    "print(x_test_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import *\n",
    "import re\n",
    "import operator\n",
    "import math\n",
    "from collections import defaultdict\n",
    "import operator\n",
    "from os import listdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    " # 创建一个包含在所有文档中出现的不重复的列表\n",
    "def create_vocab_list(data_set):\n",
    "\n",
    "    # 创建一个空集\n",
    "    vocab_set = set([])\n",
    "\n",
    "    for document in data_set:\n",
    "        vocab_set = vocab_set | set(document)\n",
    "    # 创建两个集合的并集\n",
    "    return list(vocab_set)\n",
    "\n",
    "# 切割分类文本\n",
    "def text_parse(big_string):\n",
    "    regEx = re.compile('\\\\W')  \n",
    "    list_of_tokens = regEx.split(big_string)\n",
    "#     list_of_tokens = re.split('\\W+', big_string)\n",
    "    #如果单词长度太短，就忽略该单词\n",
    "    return [tok.lower() for tok in list_of_tokens if len(tok) > 2]\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature_select(list_words):\n",
    "    #总词频统计\n",
    "    doc_frequency=defaultdict(int)\n",
    "    for word_list in list_words:\n",
    "        for i in word_list:\n",
    "            doc_frequency[i]+=1\n",
    "#     print('doc_frequency：',doc_frequency)\n",
    "    print('doc_frequency的长度：',len(doc_frequency))\n",
    "    #计算每个词的TF值\n",
    "    sum_doc_frequency=0\n",
    "    for i in doc_frequency:\n",
    "        sum_doc_frequency=sum_doc_frequency+doc_frequency[i]\n",
    "    print('sum_doc_frequency',sum_doc_frequency)\n",
    "    word_tf={}  #存储每个词的tf值\n",
    "    for i in doc_frequency:\n",
    "        word_tf[i]=doc_frequency[i]/sum_doc_frequency\n",
    "    #计算每个词的IDF值\n",
    "    doc_num=len(list_words)\n",
    "    word_idf={} #存储每个词的idf值\n",
    "    word_doc=defaultdict(int) #存储包含该词的文档数\n",
    "    for i in doc_frequency:\n",
    "        for j in list_words:\n",
    "            if i in j:\n",
    "                word_doc[i]+=1\n",
    "    for i in doc_frequency:\n",
    "        word_idf[i]=math.log(doc_num/(word_doc[i]+1))\n",
    "    #计算每个词的TF*IDF的值\n",
    "    word_tf_idf={}\n",
    "    for i in doc_frequency:\n",
    "        word_tf_idf[i]=word_tf[i]*word_idf[i]\n",
    "    # 对字典按值由大到小排序\n",
    "    dict_feature_select=sorted(word_tf_idf.items(),key=operator.itemgetter(1),reverse=True)\n",
    "    return dict_feature_select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "mylist=listdir('mini_newsgroups/')\n",
    "doc_list=[]\n",
    "dircontent=[]\n",
    "dircontent_sencond=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dircontent_sencond=listdir('mini_newsgroups/A1comp.os.ms-windows.misc/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_list=[]\n",
    "dircontent=[]\n",
    "dircontent_sencond=[]\n",
    "dircontent_sencond=listdir('mini_newsgroups/A1comp.os.ms-windows.misc/')\n",
    "\n",
    "for j in dircontent_sencond:\n",
    "    s='mini_newsgroups/A1comp.os.ms-windows.misc/%s'%j\n",
    "#     print(s)\n",
    "    f=open(s,'r')\n",
    "    txt=str(f.read())\n",
    "    txt_01=list(txt)\n",
    "    for i in range(len(txt_01)):\n",
    "        if txt_01[i].isalnum():\n",
    "            pass\n",
    "        else:\n",
    "            txt_01[i] = ' '\n",
    "        txt=''.join(txt_01)\n",
    "    doc_list.append(txt)\n",
    "    txt=''\n",
    "    f.close()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#doc_list=[]:处理好，只含有英文和数字的文本\n",
    "#doc_list=[]:每个单词用空格隔开，且过滤掉过短单词\n",
    "doc_list_join=[]\n",
    "for i in doc_list:\n",
    "    list_second=text_parse(i)\n",
    "    doc_list_join.append(' '.join(list_second))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 100)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
